{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analyze results from compare_cl_vs_vanilla"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "base = '/home/cgn/amazon_reviews/'\n",
    "results = []\n",
    "for seed in range(5):\n",
    "    for trainsize in [500000, 1000000]:\n",
    "        for epochs in [5, 20, 50]:\n",
    "            if trainsize == 500000 and epochs == 50:\n",
    "                continue\n",
    "            fn = 'out_seed_{}_trainsize_{}_epochs_{}.log'.format(\n",
    "                    seed, trainsize, epochs)\n",
    "#             print(fn)\n",
    "            with open(base + fn, 'r') as f:\n",
    "                result = f.readlines()\n",
    "                lod = eval(result[-5:-4][0].strip())\n",
    "                settings = {'seed': seed,\n",
    "                            'trainsize': trainsize,\n",
    "                            'epochs': epochs}\n",
    "                [d.update(settings) for d in lod]\n",
    "                results += lod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame(results)\n",
    "data = data[data['method'] != 'cl_intersection_all_methods']\n",
    "data.drop('train_size', axis=1, inplace=True)\n",
    "data['trainsize'] = (data['trainsize'] / 1000).round().astype(int).astype(str) + 'K'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\">$N = 1000K$</th>\n",
       "      <th colspan=\"3\" halign=\"left\">$N = 500K$</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>epochs = 5</th>\n",
       "      <th>epochs = 20</th>\n",
       "      <th colspan=\"2\" halign=\"left\">epochs = 50</th>\n",
       "      <th>epochs = 5</th>\n",
       "      <th colspan=\"2\" halign=\"left\">epochs = 20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Top-1 Acc</th>\n",
       "      <th>Top-1 Acc</th>\n",
       "      <th>Top-1 Acc</th>\n",
       "      <th>Pruned</th>\n",
       "      <th>Top-1 Acc</th>\n",
       "      <th>Top-1 Acc</th>\n",
       "      <th>Pruned</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test_split</th>\n",
       "      <th>method</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"6\" valign=\"top\">10</th>\n",
       "      <th>argmax</th>\n",
       "      <td>85.2±0.06</td>\n",
       "      <td>89.2±0.02</td>\n",
       "      <td>90.0±0.02</td>\n",
       "      <td>291K</td>\n",
       "      <td>86.6±0.03</td>\n",
       "      <td>86.6±0.03</td>\n",
       "      <td>259K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>both</th>\n",
       "      <td>86.3±0.04</td>\n",
       "      <td>89.8±0.01</td>\n",
       "      <td>90.2±0.01</td>\n",
       "      <td>250K</td>\n",
       "      <td>87.5±0.05</td>\n",
       "      <td>87.5±0.03</td>\n",
       "      <td>244K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cj_only</th>\n",
       "      <td>86.4±0.01</td>\n",
       "      <td>89.8±0.02</td>\n",
       "      <td>90.1±0.02</td>\n",
       "      <td>246K</td>\n",
       "      <td>87.5±0.02</td>\n",
       "      <td>87.5±0.02</td>\n",
       "      <td>243K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pbc</th>\n",
       "      <td>86.2±0.03</td>\n",
       "      <td>89.7±0.01</td>\n",
       "      <td>90.2±0.01</td>\n",
       "      <td>257K</td>\n",
       "      <td>87.4±0.03</td>\n",
       "      <td>87.4±0.03</td>\n",
       "      <td>247K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pbnr</th>\n",
       "      <td>86.2±0.07</td>\n",
       "      <td>89.7±0.01</td>\n",
       "      <td>90.2±0.01</td>\n",
       "      <td>257K</td>\n",
       "      <td>87.4±0.05</td>\n",
       "      <td>87.4±0.05</td>\n",
       "      <td>247K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vanilla</th>\n",
       "      <td>83.9±0.11</td>\n",
       "      <td>86.3±0.06</td>\n",
       "      <td>84.4±0.04</td>\n",
       "      <td>0K</td>\n",
       "      <td>82.7±0.07</td>\n",
       "      <td>82.8±0.07</td>\n",
       "      <td>0K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"6\" valign=\"top\">11</th>\n",
       "      <th>argmax</th>\n",
       "      <td>85.3±0.05</td>\n",
       "      <td>89.3±0.01</td>\n",
       "      <td>90.0±0.0</td>\n",
       "      <td>294K</td>\n",
       "      <td>86.6±0.04</td>\n",
       "      <td>86.6±0.06</td>\n",
       "      <td>261K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>both</th>\n",
       "      <td>86.4±0.06</td>\n",
       "      <td>89.8±0.01</td>\n",
       "      <td>90.2±0.01</td>\n",
       "      <td>252K</td>\n",
       "      <td>87.5±0.04</td>\n",
       "      <td>87.5±0.03</td>\n",
       "      <td>247K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cj_only</th>\n",
       "      <td>86.3±0.05</td>\n",
       "      <td>89.8±0.01</td>\n",
       "      <td>90.1±0.02</td>\n",
       "      <td>249K</td>\n",
       "      <td>87.5±0.03</td>\n",
       "      <td>87.5±0.02</td>\n",
       "      <td>246K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pbc</th>\n",
       "      <td>86.2±0.03</td>\n",
       "      <td>89.8±0.01</td>\n",
       "      <td>90.3±0.0</td>\n",
       "      <td>260K</td>\n",
       "      <td>87.4±0.03</td>\n",
       "      <td>87.4±0.05</td>\n",
       "      <td>250K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pbnr</th>\n",
       "      <td>86.2±0.06</td>\n",
       "      <td>89.8±0.01</td>\n",
       "      <td>90.2±0.02</td>\n",
       "      <td>260K</td>\n",
       "      <td>87.4±0.05</td>\n",
       "      <td>87.4±0.03</td>\n",
       "      <td>249K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vanilla</th>\n",
       "      <td>83.9±0.08</td>\n",
       "      <td>86.3±0.05</td>\n",
       "      <td>84.4±0.12</td>\n",
       "      <td>0K</td>\n",
       "      <td>82.7±0.04</td>\n",
       "      <td>82.7±0.09</td>\n",
       "      <td>0K</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   $N = 1000K$                                $N = 500K$  \\\n",
       "                    epochs = 5 epochs = 20 epochs = 50        epochs = 5   \n",
       "                     Top-1 Acc   Top-1 Acc   Top-1 Acc Pruned  Top-1 Acc   \n",
       "test_split method                                                          \n",
       "10         argmax    85.2±0.06   89.2±0.02   90.0±0.02   291K  86.6±0.03   \n",
       "           both      86.3±0.04   89.8±0.01   90.2±0.01   250K  87.5±0.05   \n",
       "           cj_only   86.4±0.01   89.8±0.02   90.1±0.02   246K  87.5±0.02   \n",
       "           pbc       86.2±0.03   89.7±0.01   90.2±0.01   257K  87.4±0.03   \n",
       "           pbnr      86.2±0.07   89.7±0.01   90.2±0.01   257K  87.4±0.05   \n",
       "           vanilla   83.9±0.11   86.3±0.06   84.4±0.04     0K  82.7±0.07   \n",
       "11         argmax    85.3±0.05   89.3±0.01    90.0±0.0   294K  86.6±0.04   \n",
       "           both      86.4±0.06   89.8±0.01   90.2±0.01   252K  87.5±0.04   \n",
       "           cj_only   86.3±0.05   89.8±0.01   90.1±0.02   249K  87.5±0.03   \n",
       "           pbc       86.2±0.03   89.8±0.01    90.3±0.0   260K  87.4±0.03   \n",
       "           pbnr      86.2±0.06   89.8±0.01   90.2±0.02   260K  87.4±0.05   \n",
       "           vanilla   83.9±0.08   86.3±0.05   84.4±0.12     0K  82.7±0.04   \n",
       "\n",
       "                                       \n",
       "                   epochs = 20         \n",
       "                     Top-1 Acc Pruned  \n",
       "test_split method                      \n",
       "10         argmax    86.6±0.03   259K  \n",
       "           both      87.5±0.03   244K  \n",
       "           cj_only   87.5±0.02   243K  \n",
       "           pbc       87.4±0.03   247K  \n",
       "           pbnr      87.4±0.05   247K  \n",
       "           vanilla   82.8±0.07     0K  \n",
       "11         argmax    86.6±0.06   261K  \n",
       "           both      87.5±0.03   247K  \n",
       "           cj_only   87.5±0.02   246K  \n",
       "           pbc       87.4±0.05   250K  \n",
       "           pbnr      87.4±0.03   249K  \n",
       "           vanilla   82.7±0.09     0K  "
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols = ['Top-1 Acc', 'Pruned']\n",
    "df = data.groupby(['test_split', 'trainsize', 'epochs', 'method']).agg(['mean', 'std'])\n",
    "# df.columns = [' '.join(col).strip() for col in df.columns.values]\n",
    "df.drop([('seed', 'mean'), ('seed', 'std')], axis=1, inplace=True)\n",
    "# Formatting\n",
    "df[('data_removed', 'mean')] = (df[('data_removed', 'mean')] / 1000).round().astype(int).astype(str) + 'K'\n",
    "# df[('data_removed', 'std')] = df[('data_removed', 'std')].round().astype(int)\n",
    "del df[('data_removed', 'std')]\n",
    "df[('acc', 'mean')] = (df[('acc', 'mean')] * 100).round(1)\n",
    "df[('acc', 'std')] = (df[('acc', 'std')] * 100).round(2)\n",
    "df.columns = [' '.join(col).strip() for col in df.columns.values]\n",
    "df['acc'] = df['acc mean'].astype(str) + '±' + df['acc std'].astype(str)\n",
    "del df['acc mean']\n",
    "del df['acc std']\n",
    "df = df[['acc', 'data_removed mean']]\n",
    "df.columns = cols\n",
    "# pd.concat([z for i, z in df.reset_index().groupby(['test_split', 'trainsize'])])\n",
    "sdfs = []\n",
    "for key, sdf in df.reset_index(level=[1,2]).groupby(['trainsize', 'epochs']):\n",
    "    z = sdf[cols]\n",
    "    label1 = r'$N = {}$'.format(key[0])\n",
    "    label2 = 'epochs = {}'.format(key[1])\n",
    "    z.columns = pd.MultiIndex.from_product([[label1], [label2], z.columns])\n",
    "    sdfs.append(z)\n",
    "paper_df = pd.concat(sdfs, axis=1)\n",
    "for c in [('$N = 1000K$',  'epochs = 5',    'Pruned'),\n",
    "          ('$N = 1000K$', 'epochs = 20',    'Pruned'),\n",
    "          ('$N = 500K$',  'epochs = 5',     'Pruned'),]:\n",
    "    del paper_df[c]\n",
    "paper_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "method_name_map = {\n",
    "\t'argmax': r'CL: $\\bm{C}_{\\text{confusion}}$',\n",
    "\t'pbc': 'CL: PBC',\n",
    "\t'cj\\_only': r'CL: $\\cj$',\n",
    "\t'both': 'CL: C+NR',\n",
    "\t'pbnr': 'CL: PBNR',\n",
    "\t'vanilla': 'Baseline',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lllllllll}\n",
      "\\toprule\n",
      "   &         & \\multicolumn{4}{l}{$N = 1000K$} & \\multicolumn{3}{l}{$N = 500K$} \\\\\n",
      "   &         &  epochs = 5 & epochs = 20 & \\multicolumn{2}{l}{epochs = 50} & epochs = 5 & \\multicolumn{2}{l}{epochs = 20} \\\\\n",
      "   &         &   Top-1 Acc &   Top-1 Acc &   Top-1 Acc & Pruned &  Top-1 Acc &   Top-1 Acc & Pruned \\\\\n",
      "test\\_split & method &             &             &             &        &            &             &        \\\\\n",
      "\\midrule\n",
      "10 & CL: $\\bm{C}_{\\text{confusion}}$ &   85.2$\\pm$0.06 &   89.2$\\pm$0.02 &   90.0$\\pm$0.02 &   291K &  86.6$\\pm$0.03 &   86.6$\\pm$0.03 &   259K \\\\\n",
      "   & CL: C+NR &   86.3$\\pm$0.04 &   89.8$\\pm$0.01 &   90.2$\\pm$0.01 &   250K &  87.5$\\pm$0.05 &   87.5$\\pm$0.03 &   244K \\\\\n",
      "   & CL: $\\cj$ &   86.4$\\pm$0.01 &   89.8$\\pm$0.02 &   90.1$\\pm$0.02 &   246K &  87.5$\\pm$0.02 &   87.5$\\pm$0.02 &   243K \\\\\n",
      "   & CL: PBC &   86.2$\\pm$0.03 &   89.7$\\pm$0.01 &   90.2$\\pm$0.01 &   257K &  87.4$\\pm$0.03 &   87.4$\\pm$0.03 &   247K \\\\\n",
      "   & CL: PBNR &   86.2$\\pm$0.07 &   89.7$\\pm$0.01 &   90.2$\\pm$0.01 &   257K &  87.4$\\pm$0.05 &   87.4$\\pm$0.05 &   247K \\\\\n",
      "   & Baseline &   83.9$\\pm$0.11 &   86.3$\\pm$0.06 &   84.4$\\pm$0.04 &     0K &  82.7$\\pm$0.07 &   82.8$\\pm$0.07 &     0K \\\\\n",
      "11 & CL: $\\bm{C}_{\\text{confusion}}$ &   85.3$\\pm$0.05 &   89.3$\\pm$0.01 &    90.0$\\pm$0.0 &   294K &  86.6$\\pm$0.04 &   86.6$\\pm$0.06 &   261K \\\\\n",
      "   & CL: C+NR &   86.4$\\pm$0.06 &   89.8$\\pm$0.01 &   90.2$\\pm$0.01 &   252K &  87.5$\\pm$0.04 &   87.5$\\pm$0.03 &   247K \\\\\n",
      "   & CL: $\\cj$ &   86.3$\\pm$0.05 &   89.8$\\pm$0.01 &   90.1$\\pm$0.02 &   249K &  87.5$\\pm$0.03 &   87.5$\\pm$0.02 &   246K \\\\\n",
      "   & CL: PBC &   86.2$\\pm$0.03 &   89.8$\\pm$0.01 &    90.3$\\pm$0.0 &   260K &  87.4$\\pm$0.03 &   87.4$\\pm$0.05 &   250K \\\\\n",
      "   & CL: PBNR &   86.2$\\pm$0.06 &   89.8$\\pm$0.01 &   90.2$\\pm$0.02 &   260K &  87.4$\\pm$0.05 &   87.4$\\pm$0.03 &   249K \\\\\n",
      "   & Baseline &   83.9$\\pm$0.08 &   86.3$\\pm$0.05 &   84.4$\\pm$0.12 &     0K &  82.7$\\pm$0.04 &   82.7$\\pm$0.09 &     0K \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tex = paper_df.to_latex().replace('\\\\$', '$').replace('±', '$\\pm$')\n",
    "tex = ' '.join([method_name_map. get(i, i) for i in tex.split(' ')])\n",
    "print(tex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
